﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace WinFormsCrawler
{
    public class Crawler
    {
        public Hashtable urls = new Hashtable();
        private int count = 0;

        public bool _IsRunning = false;

        //初始爬取地址
        public string InitURL { get; internal set; }

        public void Start()
        {
            _IsRunning = true;
            LogInfo.Instance.Log("开始爬行了。。。");
            while (_IsRunning)
            {
                string current = null;
                foreach (string url in urls.Keys)  //找到一个还没有下载过的链接
                {
                    if ((bool)urls[url]) continue; //已经下载过的,不再下载
                    current = url;
                }

                if (current == null || count > 10) break;

                LogInfo.Instance.Log("爬行" + current + "页面。。。");

                string html = DownLoad(current); //下载
                urls[current] = true;
                count++;

                Parse(current, html); //解析,并加入新的链接

            }
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webclient = new WebClient();
                webclient.Encoding = Encoding.UTF8;
                string html = webclient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch(Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }

        public void Parse(string current, string html)
        {
            //判定是Html文本
            if (!IsHtmlText(html)) return;

            //string strRef = @"(href|HREF)[]* = []*[""'][^""'#>] + [""']";
            string strRef = @"href\s*=\s*(?:""([^""]*)""|'([^']*)'|(\S+))";
            MatchCollection matches = new Regex(strRef).Matches(html);

            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"','\"','#',' ','>');
                if (strRef.Length == 0) continue;
                if (strRef.Length > 1 && strRef[0] == '/')
                {
                    strRef = current + strRef;
                }

                //判定是否为初始地址
                if (FindBaseURL(strRef) == FindBaseURL(InitURL))
                {
                    if (urls[strRef] == null) urls[strRef] = false;
                }

            }
        }

        bool IsHtmlText(string text)
        {
            // 正则表达式匹配HTML标签
            string pattern = @"<(\w+)\b.*?>";
            return Regex.IsMatch(text, pattern);
        }

        string FindBaseURL(string url)
        {
            int index = url.IndexOf("//");
            if (index > 0)
            {
                url = url.Substring(index + 2);
                index = url.IndexOf("/");
                if (index > 0) url = url.Remove(index);
                return url;
            }
            return null;
        }
    }
}
